from research.frame import allk1d
import os
from jili.core import save,load
from jili.data.db import getdb_client
from jili.data.db import get_calender_a
import pandas as pd
from tqdm import tqdm
from jili.tool.convert import str2datetime
from copy import deepcopy
from research.factor import gen_tradedata_a
def getsave_index_detail_byday(save_url,ip="127.0.0.1",start_date=None,end_date=None,level="L2",mode="0",cu_level="index_calssify_sw_level",cu_detail="index_calssify_sw_detail"):
    db = getdb_client("stock", ip)
    cu=db[cu_detail]
    sw={}
    rst = {}
    detail = {}
    for i in db[cu_level].find({"level":level}):
        id=i["index_code"]
        if "index_name" in i.keys():
            sw[id] = i["index_name"]
        else:
            sw[id]=i["industry_name"]
    min_date = None
    if mode=="0":
        for i in cu.find({}):
            del i["_id"]
            index = i["index_code"]
            date=i["date"]
            if index in sw.keys():
                if index not in detail.keys():
                    detail[index]={}
                if date not in detail[index].keys():
                    detail[index][date]=[]
                detail[index][date].append(i)
                indate = "date"
                if min_date:
                    if min_date > i[indate]:
                        min_date = i[indate]
                else:
                    min_date = i[indate]
        if start_date:
            if str2datetime(min_date) < str2datetime(start_date):
                min_date = start_date
        detaild={}
        for index_code, v in detail.items():
            l=list(v.keys())
            l.sort()
            ll = []
            for date in l:
                l1 = []
                for i in v[date]:
                    action = i["action"]
                    obj = i["obj"]
                    if action == "纳入":
                        if obj not in ll:
                            ll.append(obj)
                    elif action == "剔除":
                        if obj not in l1:
                            l1.append(obj)
                for i in l1:
                    if i not in ll:
                        print(index_code,date,i,"无纳入")
                    else:
                        ll.remove(i)
                if index_code not in detaild.keys():
                    detaild[index_code]={}
                detaild[index_code][date]=deepcopy(ll)
        rst=gen_tradedata_a(detaild,end_date)
    else:
        for i in cu.find({}):
            del i["_id"]
            index = i["index_code"]
            if index in sw.keys():
                if index in detail.keys():
                    detail[index].append(i)
                else:
                    detail[index] = [i]
                if "in_date" not in i.keys():
                    indate = "date"
                else:
                    indate = "in_date"

                if min_date:
                    if min_date > i[indate]:
                        min_date = i[indate]
                else:
                    min_date = i[indate]
        if start_date:
            if str2datetime(min_date) < str2datetime(start_date):
                min_date = start_date
        for d in tqdm(get_calender_a(start=min_date,end=end_date)):
            date=d.strftime("%Y%m%d")
            for index_code,v in detail.items():
                ll=[]
                for i in v:
                    in_date=i["in_date"]
                    out_date=i["out_date"]
                    obj=i["obj"]
                    if out_date:
                        if in_date<=date and date<=out_date:
                            if obj not in ll:
                                ll.append(obj)
                    else:
                        if in_date<=date:
                            if obj not in ll:
                                ll.append(obj)
                if ll:
                    if d not in rst.keys():
                        rst[d]={index_code:ll}
                    else:
                        t=rst[d]
                        t[index_code]=ll
    save(rst,save_url)
    return rst
def deal_sw_objs_byday(ip="127.0.0.1",level="L2",cu_level="index_calssify_sw_level",cu_detail="index_calssify_sw_detail"):
    db = getdb_client("stock", ip)
    cu=db[cu_detail]
    sw={}
    for i in db[cu_level].find({"level":level}):
        id=i["index_code"]
        sw[id]=i["industry_name"]
    detail = {}
    min_date=None
    for i in cu.find({}):
        del i["_id"]
        index=i["index_code"]
        if index in sw.keys():
            if index in detail.keys():
                detail[index].append(i)
            else:
                detail[index]=[i]
            if min_date:
                if min_date>i["in_date"]:
                    min_date=i["in_date"]
            else:
                min_date=i["in_date"]
    rst={}
    for d in tqdm(get_calender_a(start=min_date)):
        date=d.strftime("%Y%m%d")
        for index_code,v in detail.items():
            ll=[]
            for i in v:
                in_date=i["in_date"]
                out_date=i["out_date"]
                obj=i["obj"]
                if out_date:
                    if in_date<=date and date<=out_date:
                        if obj not in ll:
                            ll.append(obj)
                else:
                    if in_date<=date:
                        if obj not in ll:
                            ll.append(obj)
            if ll:
                if d not in rst.keys():
                    rst[d]={index_code:ll}
                else:
                    t=rst[d]
                    t[index_code]=ll
    return rst
def calc_new_theme_concept_wind_idnex():
    wind_theme_index = load(r"G:\factor\k1d\is_wind_detail_l1.pkl")
    wind_concept_index = load(r"G:\factor\k1d\is_wind_concept_detail_l1.pkl")
    add_theme_index_list = ['884039.WI','884045.WI']
    for date in tqdm(wind_theme_index.keys()):
        v = wind_theme_index[date]
        if date in wind_concept_index.keys():
            date_concept = wind_concept_index[date]
            for i in add_theme_index_list:
                if i in date_concept.keys():
                    wind_theme_index[date][i] = date_concept[i]
                    for stock in date_concept[i]:
                        for theme,v1 in v.items():
                            if (theme != i) & (stock in v1):
                                # print(date,theme,stock)
                                v1.remove(stock)
                                wind_theme_index[date][theme] = v1
    save(wind_theme_index,r"G:\factor\k1d\is_wind_new_theme_detail.pkl")
def calc_index_move2(detail,rps,name="move_rps",level=87):
    rpsd=load(os.path.join(r"G:\factor\k1d\new1",rps))
    detaild=load(os.path.join(r"G:\factor\k1d\new1",detail))
    rst={}
    jg=load(r"G:\factor\k1d\new1\top10_floatholders_jg")
    trade_days=load(r"G:\factor\k1d\new1\trade_days")
    for d,v in tqdm(detaild.items()):
        if d in rpsd.keys():
            if d not in rst.keys():
                rst[d]={}
            t=rst[d]
            for index,objs in v.items():
                objs0=[]
                topn=0

                for obj in objs:
                    if d in trade_days.keys():
                        if obj in trade_days[d].keys():
                            if trade_days[d][obj]<=120:
                                continue
                    if d in jg.keys():
                        if obj in jg[d].keys():
                            if jg[d][obj]<=0:
                                continue
                    if obj in rpsd[d].keys():
                        objs0.append(obj)
                        if rpsd[d][obj]>=level:
                            topn=topn+1
                if topn==0:
                    t0 = {name: 0}
                else:
                    t0={name:topn**2/len(objs0)}
                t[index]=t0
    rst1={}
    for d,v in rst.items():
        k = pd.DataFrame(v).T
        k.sort_values(["move_rps"], ascending=False, inplace=True)
        n = len(k)
        k["rank"] = range(1, n + 1, 1)
        k["rps"]=100*(n-k["rank"])/n
        rst1[d]=k.to_dict("index")
    return rst1
def calc_index_move1(detail,rps,name="move_rps",level=87):
    rpsd=load(os.path.join(r"G:\factor\k1d\new1",rps))
    detaild=load(os.path.join(r"G:\factor\k1d\new1",detail))
    rst={}
    jg=load(r"G:\factor\k1d\new1\top10_floatholders_jgrate")
    trade_days=load(r"G:\factor\k1d\new1\trade_days")
    for d,v in tqdm(detaild.items()):
        if d in rpsd.keys():
            if d not in rst.keys():
                rst[d]={}
            t=rst[d]
            for index,objs in v.items():
                objs0=[]
                topn=0

                for obj in objs:
                    if d in trade_days.keys():
                        if obj in trade_days[d].keys():
                            if trade_days[d][obj]<=120:
                                continue
                    if d in jg.keys():
                        if obj in jg[d].keys():
                            if jg[d][obj]<0.02:
                                continue
                    if obj in rpsd[d].keys():
                        objs0.append(obj)
                        if rpsd[d][obj]>=level:
                            topn=topn+1
                if topn==0:
                    t0 = {name: 0}
                else:
                    t0={name:topn**2/len(objs0)}
                t[index]=t0
    rst1={}
    for d,v in rst.items():
        k = pd.DataFrame(v).T
        k.sort_values(["move_rps"], ascending=False, inplace=True)
        n = len(k)
        k["rank"] = range(1, n + 1, 1)
        k["rps"]=100*(n-k["rank"])/n
        rst1[d]=k.to_dict("index")
    return rst1
def calc_index_strength(detail,rps,save_url="strength_score",level=87,ip="192.168.10.44"):
    rpsd=load(os.path.join(r"G:\factor\k1d",rps))
    trade_days = load(os.path.join(r"G:\factor\k1d","trade_days_old"))
    detaild=load(os.path.join(r"G:\factor\k1d",detail))
    #top10_holders_jgrate = load(os.path.join(r"G:\factor\k1d", "top10_holders_jgrate.pkl"))
    top10_holders_jgrate = load(r'G:\factor\k1d\jg_hold_info')
    #close_p = load(os.path.join(r"G:\factor\k1d","tech_close"))
    #ma_close_150 = load(os.path.join(r"G:\factor\k1d", "ma_close_150"))
#    db = getdb_client("stock", ip)
#    cu = db['index_calssify_dc_level']
#    ind_info = {}
#    for i in cu.find():
#        if i['index_code'] not in ind_info.keys():
#            ind_info[i['index_code']] = i['industry_name']
    rst={}
    for d,v in tqdm(detaild.items()):
        #if d != datetime.datetime(2021, 8, 6):
        #    continue
        if (d in rpsd.keys()) & (d in trade_days.keys()):
            if d not in rst.keys():
                rst[d]={}
            t=rst[d]
            for index,objs in v.items():
                topn=0
                for obj in objs:
                    if obj in rpsd[d].keys():
                        if rpsd[d][obj]>=level:
                            if trade_days[d][obj] >= 180:
                                try:
                                    if top10_holders_jgrate[d][obj]["freehold_ratio"] >= 0.02:
                                        topn = topn + 1
                                except:
                                    pass
                t0=topn**2/len(objs)
                #t[ind_info[index]]=t0
                t[index] = t0
    #rst1={}
    #for d,v in rst.items():
    #    k = pd.DataFrame(v).T
    #    k.sort_values(["move_rps"], ascending=False, inplace=True)
    #    n = len(k)
    #    k["rank"] = range(1, n + 1, 1)
    #    k["rps"]=100*(n-k["rank"])/n
    #    rst1[d]=k.to_dict("index")
    save(rst, os.path.join(r"G:\factor\k1d",save_url))
    return rst
def calc_index(mode="avg",urlk1d=r"G:\factor\k1d\kline_k1d.pkl",indexdetail=r"G:\factor\k1d\sw_l3_detail",name="sw_l3_k1d"):
    """
    mode:avg,weight
    """
    sw=load(indexdetail)
    startdate=None
    rst={}
    for k in sw.keys():
        if not startdate:
            startdate=k
            break
    k1d=load(urlk1d)
    for k,v in tqdm(k1d.items()):
        if k>=startdate:
            detail=sw[k]
            for index_code ,objs in detail.items():
                n=len(objs)
                t0=0
                for obj in objs:
                    if obj in v.keys():
                        bar=v[obj]
                        pct=bar["pctChg"]
                        if mode=="avg":
                            t0=t0+pct/n
                        else:
                            t0=t0+pct/n
                t={"obj":index_code,"timekey":k,"pctChg":t0}
                if index_code not in rst.keys():
                    rst[index_code]=[t]
                else:
                    rst[index_code].append(t)
    return rst
def deal_NHNL(url0=r"G:\factor\k1d\new1\new_high_days_byhigh",url1=r"G:\factor\k1d\new1\new_low_days_bylow",save_url=r"G:\factor\k1d\new1\nhnl_byhl",days=60):
    a=load(url0)
    b=load(url1)
    rst={}
    for date,v in a.items():
        nh=0
        for obj,v1 in v.items():
            if v1>=days:
                nh=nh+1
        nl=0
        for obj,v1 in b[date].items():
            if v1>=days:
                nl=nl+1
        rst[date]=nh-nl
    save(rst,save_url)
def deal_count_factor(url=r"G:\factor\k1d\new1\new_high_days_byhigh",save_url=r"G:\factor\k1d\new1\nhnl_byhl",days=60):
    a=load(url)
    rst={}
    for date,v in a.items():
        nh=0
        for obj,v1 in v.items():
            if v1>=days:
                nh=nh+1
        rst[date]=nh
    save(rst,save_url)

def move_rps2(n=95):
    a = calc_index_move2("is_sw_detail_l1", "rps_20", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_sw_l1_move_rps_20_" + str(n))
    a = calc_index_move2("is_sw_detail_l1", "rps_50", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_sw_l1_move_rps_50_" + str(n))
    a = calc_index_move2("is_sw_detail_l1", "rps_10", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_sw_l1_move_rps_10_" + str(n))
    a = calc_index_move2("is_sw_detail_l2", "rps_20", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_sw_l2_move_rps_20_" + str(n))
    a = calc_index_move2("is_sw_detail_l2", "rps_50", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_sw_l2_move_rps_50_" + str(n))

    a = calc_index_move2("is_dc_detail_l1", "rps_50", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_dc_l1_move_rps_50_" + str(n))
    a = calc_index_move2("is_dc_detail_l1", "rps_10", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_dc_l1_move_rps_10_" + str(n))
    a = calc_index_move2("is_dc_detail_l2", "rps_20", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_dc_l2_move_rps_20_" + str(n))
    a = calc_index_move2("is_dc_detail_l2", "rps_50", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_dc_l2_move_rps_50_" + str(n))

    a = calc_index_move2("is_sw_detail_l2", "rps_10", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_sw_l2_move_rps_10_" + str(n))
    a = calc_index_move2("is_dc_detail_l1", "rps_20", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_dc_l1_move_rps_20_" + str(n))
    a = calc_index_move2("is_dc_detail_l2", "rps_10", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_dc_l2_move_rps_10_" + str(n))
def move_rps(n=95):
    a=calc_index_move1("is_sw_detail_l1", "rps_20", name="move_rps", level=n )
    save(a, r"G:\factor\k1d\new1\index_sw_l1_move_rps_20_"+str(n))
    a = calc_index_move1("is_sw_detail_l1", "rps_50", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_sw_l1_move_rps_50_"+str(n))
    a = calc_index_move1("is_sw_detail_l1", "rps_10", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_sw_l1_move_rps_10_"+str(n))
    a = calc_index_move1("is_sw_detail_l2", "rps_20", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_sw_l2_move_rps_20_"+str(n))
    a = calc_index_move1("is_sw_detail_l2", "rps_50", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_sw_l2_move_rps_50_"+str(n))

    a = calc_index_move1("is_dc_detail_l1", "rps_50", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_dc_l1_move_rps_50_"+str(n))
    a = calc_index_move1("is_dc_detail_l1", "rps_10", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_dc_l1_move_rps_10_"+str(n))
    a = calc_index_move1("is_dc_detail_l2", "rps_20", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_dc_l2_move_rps_20_"+str(n))
    a = calc_index_move1("is_dc_detail_l2", "rps_50", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_dc_l2_move_rps_50_"+str(n))

    a = calc_index_move1("is_sw_detail_l2", "rps_10", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_sw_l2_move_rps_10_"+str(n))
    a = calc_index_move1("is_dc_detail_l1", "rps_20", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_dc_l1_move_rps_20_"+str(n))
    a = calc_index_move1("is_dc_detail_l2", "rps_10", name="move_rps", level=n)
    save(a, r"G:\factor\k1d\new1\index_dc_l2_move_rps_10_"+str(n))
def deal_index():
    getsave_index_detail_byday(save_url=r"G:\factor\k1d\is_wind_concept_detail_l1", ip="127.0.0.1", end_date="20210902",
                               level="L1", mode="0", cu_level="index_calssify_wind_concept_level",
                               cu_detail="index_calssify_wind_concept_detail")
    getsave_index_detail_byday(save_url=r"G:\factor\k1d\is_wind_detail_l1", ip="127.0.0.1", end_date="20210902",
                               level="L1", mode="0", cu_level="index_calssify_wind_level",
                               cu_detail="index_calssify_wind_detail")

def index_weight_daily(index_code = '000300',ip="192.168.10.44",mode = '0',start_date = None,save_url= r"G:\factor\k1d\index_weight"):
    db = getdb_client("stock", ip)
    cu = db['index_weight']
    index_weight = {}
    detail = {}
    min_date = None
    max_date = None
    if mode == "0":
        for i in cu.find({}):
            del i["_id"]
            index = i["obj"]
            date = i["date"]
            if index == index_code:
                if date not in detail.keys():
                    detail[date] = []
                detail[date].append(i)
                indate = "date"
                if min_date:
                    if min_date > i[indate]:
                        min_date = i[indate]
                else:
                    min_date = i[indate]
                if max_date:
                    if max_date < i[indate]:
                        max_date = i[indate]
                else:
                    max_date = i[indate]
        if start_date:
            if str2datetime(min_date) < str2datetime(start_date):
                min_date = start_date
        detaild = {}
        date_list = list(detail.keys())
        date_list.sort()
        for date in date_list:
            l1 = []
            ll = []
            for i in detail[date]:
                action = i["action"]
                obj = i["obj_code"]
                if action == "纳入":
                    if obj not in ll:
                        ll.append(obj)
                elif action == "剔除":
                    if obj not in l1:
                        l1.append(obj)
            for i in l1:
                if i not in ll:
                    print(index_code, date, i, "无纳入")
                else:
                    ll.remove(i)
            if index_code not in detaild.keys():
                detaild[index_code] = {}
            detaild[index_code][date] = deepcopy(ll)
        rst = gen_tradedata_a(detaild, max_date)
    save(rst, save_url + '_' + index_code)

def market_sentiment_index(url=r"G:\factor\k1d",short_rps = 'rps_5',long_rps = 'rps_50',longrps_limit = 90,
                           save_url= r"G:\factor\k1d\market_sentiment_index"):
    srps = load(os.path.join(url, short_rps))
    lrps = load(os.path.join(url, long_rps))
    result = {}
    for date,v in lrps.items():
        srps_date = pd.Series(srps[date])
        v = pd.Series(v)
        stocklist = list(v[v>longrps_limit].index)
        result[date] = srps_date.loc[stocklist].mean()
    save(result,save_url)
if __name__ == '__main__':
    # from research.factor import factor_hisbatch_bygrap
    # from research.factor.gegu import deal_rps_50
    # from research.frame.allk1d import get_bardict
    #""":cvar
    #a=get_bardict("1990","20210401")
    #save(a,r"G:\factor\k1d\kline_k1d.pkl")
    #a=deal_sw_objs_byday()
    #save(a, r"G:\factor\k1d\sw_l3_detail")
    # a = calc_index(mode="avg",urlk1d=r"G:\factor\k1d\kline_k1d.pkl",indexdetail=r"G:\factor\k1d\sw_l3_detail",name="sw_l3_k1d")
    # index_rps={}
    # for obj,v in a.items():
    #     save(v, os.path.join(r"G:\factor\k1d\index",obj+"_k1d"))
    #     factors = {
    #         "pctchg_50": {'calc_cmd': 'ta', 'cmd': 'SUM', 'out': ['pctchg_50'],
    #                       'input': {'price': 'pctChg'},
    #                       'arg': {'timeperiod': 50}, 'batch': 50},
    #     }
    #     f = factor_hisbatch_bygrap()
    #     rps=f.calc_graph(v,factors["pctchg_50"])
    #     for d,s in rps.items():
    #         for obj,t in s.items():
    #             if d in index_rps.keys():
    #                 tt=index_rps[d]
    #                 tt[obj]=t
    #             else:
    #                 index_rps[d]=s
    # save(index_rps, os.path.join(r"G:\factor\k1d\index_sw_pctchg_50"))
    
    #"""
    # deal_NHNL(url0=r"G:\factor\k1d\new1\new_high_days", url1=r"G:\factor\k1d\new1\new_low_days",
    #           save_url=r"G:\factor\k1d\new1\nhnl_60", days=60)
    # deal_NHNL(url0=r"G:\factor\k1d\new1\new_high_days", url1=r"G:\factor\k1d\new1\new_low_days",
    #           save_url=r"G:\factor\k1d\new1\nhnl_120", days=120)
    # deal_NHNL(url0=r"G:\factor\k1d\new1\new_high_days_byhigh", url1=r"G:\factor\k1d\new1\new_low_days_bylow",
    #           save_url=r"G:\factor\k1d\new1\nhnl_byhl_120", days=120)
    # deal_NHNL(url0=r"G:\factor\k1d\new1\new_high_days_byhigh", url1=r"G:\factor\k1d\new1\new_low_days_bylow",
    #           save_url=r"G:\factor\k1d\new1\nhnl_byhl_60", days=60)
    # deal_count_factor(url=r"G:\factor\k1d\new1\new_high_days_byhigh", save_url=r"G:\factor\k1d\new1\nh_byhigh_60", days=60)
    # deal_count_factor(url=r"G:\factor\k1d\new1\new_high_days_byhigh", save_url=r"G:\factor\k1d\new1\nh_byhigh_120", days=120)
    # deal_count_factor(url=r"G:\factor\k1d\new1\new_high_days", save_url=r"G:\factor\k1d\new1\nh_60", days=60)
    # deal_count_factor(url=r"G:\factor\k1d\new1\new_high_days", save_url=r"G:\factor\k1d\new1\nh_120", days=120)
    # deal_count_factor(url=r"G:\factor\k1d\new1\new_low_days_bylow", save_url=r"G:\factor\k1d\new1\nl_bylow_60", days=60)
    # deal_count_factor(url=r"G:\factor\k1d\new1\new_low_days_bylow", save_url=r"G:\factor\k1d\new1\nl_bylow_120", days=120)
    # deal_count_factor(url=r"G:\factor\k1d\new1\new_low_days", save_url=r"G:\factor\k1d\new1\nl_60", days=60)
    # deal_count_factor(url=r"G:\factor\k1d\new1\new_low_days", save_url=r"G:\factor\k1d\new1\nl_120", days=120)
    #
    # a=deal_sw_objs_byday(ip="127.0.0.1",level="L2",cu_detail="index_calssify_sw_detail")
    # save(a,r"G:\factor\k1d\new1\is_sw_detail_l2")
    # a = deal_sw_objs_byday(ip="127.0.0.1", level="L1", cu_detail="index_calssify_sw_detail")
    # save(a, r"G:\factor\k1d\new1\is_sw_detail_l1")
    # a = deal_sw_objs_byday(ip="127.0.0.1", level="L2",cu_level="index_calssify_dc_level", cu_detail="index_calssify_dc_detail")
    # save(a, r"G:\factor\k1d\new1\is_dc_detail_l2")
    # a = deal_sw_objs_byday(ip="127.0.0.1", level="L1",cu_level="index_calssify_dc_level", cu_detail="index_calssify_dc_detail")
    # save(a, r"G:\factor\k1d\new1\is_dc_detail_l1")
    #move_rps(87)
    deal_index()
    # move_rps(90)
    # move_rps(92)
    # move_rps(95)








